<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>Painless examples for transforms | Elasticsearch Guide [7.7] | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch Guide [7.7]">
<link rel="up" href="transforms.html" title="Transforming data">
<link rel="prev" href="transform-examples.html" title="Transform examples">
<link rel="next" href="transform-troubleshooting.html" title="Troubleshooting transforms">
<meta name="DC.type" content="Learn/Docs/Elasticsearch/Reference/7.7">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="7.7">
<meta name="robots" content="noindex,nofollow">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<strong>IMPORTANT</strong>: No additional bug fixes or documentation updates
will be released for this version. For the latest information, see the
<a href="../current/index.html">current release documentation</a>.
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch Guide [7.7]</a></span>
»
<span class="breadcrumb-link"><a href="data-rollup-transform.html">Roll up or transform your data</a></span>
»
<span class="breadcrumb-link"><a href="transforms.html">Transforming data</a></span>
»
<span class="breadcrumb-node">Painless examples for transforms</span>
</div>
<div class="navheader">
<span class="prev">
<a href="transform-examples.html">« Transform examples</a>
</span>
<span class="next">
<a href="transform-troubleshooting.html">Troubleshooting transforms »</a>
</span>
</div>
<div class="section xpack">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="transform-painless-examples"></a>Painless examples for transforms<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/transform/painless-examples.asciidoc">edit</a><a class="xpack_tag" href="/subscriptions"></a>
</h2>
</div></div></div>

<p>These examples demonstrate how to use Painless in transforms. You can learn
more about the Painless scripting language in the
<a href="/guide/en/elasticsearch/painless/7.7/painless-guide.html" class="ulink" target="_top">Painless guide</a>.</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
<a class="xref" href="transform-painless-examples.html#painless-top-hits" title="Getting top hits by using scripted metric aggregation">Getting top hits by using scripted metric aggregation</a>
</li>
<li class="listitem">
<a class="xref" href="transform-painless-examples.html#painless-time-features" title="Getting time features by using aggregations">Getting time features by using aggregations</a>
</li>
<li class="listitem">
<a class="xref" href="transform-painless-examples.html#painless-group-by" title="Using Painless in group_by">Using Painless in <code class="literal">group_by</code></a>
</li>
<li class="listitem">
<a class="xref" href="transform-painless-examples.html#painless-bucket-script" title="Getting duration by using bucket script">Getting duration by using bucket script</a>
</li>
<li class="listitem">
<a class="xref" href="transform-painless-examples.html#painless-count-http" title="Counting HTTP responses by using scripted metric aggregation">Counting HTTP responses by using scripted metric aggregation</a>
</li>
<li class="listitem">
<a class="xref" href="transform-painless-examples.html#painless-compare" title="Comparing indices by using scripted metric aggregations">Comparing indices by using scripted metric aggregations</a>
</li>
<li class="listitem">
<a class="xref" href="transform-painless-examples.html#painless-web-session" title="Getting web session details by using scripted metric aggregation">Getting web session details by using scripted metric aggregation</a>
</li>
</ul>
</div>
<div class="note admon">
<div class="icon"></div>
<div class="admon_content">
<p>While the context of the following examples is the transform use case,
the Painless scripts in the snippets below can be used in other Elasticsearch search
aggregations, too.</p>
</div>
</div>
<h4>
<a id="painless-top-hits"></a>Getting top hits by using scripted metric aggregation<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/transform/painless-examples.asciidoc">edit</a>
</h4>
<p>This snippet shows how to find the latest document, in other words the document
with the earliest timestamp. From a technical perspective, it helps to achieve
the function of a <a class="xref" href="search-aggregations-metrics-top-hits-aggregation.html" title="Top Hits Aggregation">Top Hits Aggregation</a> by using
scripted metric aggregation in a transform, which provides a metric output.</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">"aggregations": {
  "latest_doc": {
    "scripted_metric": {
      "init_script": "state.timestamp_latest = 0L; state.last_doc = ''", <a id="CO448-1"></a><i class="conum" data-value="1"></i>
      "map_script": """ <a id="CO448-2"></a><i class="conum" data-value="2"></i>
        def current_date = doc['@timestamp'].getValue().toInstant().toEpochMilli();
        if (current_date &gt; state.timestamp_latest)
        {state.timestamp_latest = current_date;
        state.last_doc = new HashMap(params['_source']);}
      """,
      "combine_script": "return state", <a id="CO448-3"></a><i class="conum" data-value="3"></i>
      "reduce_script": """ <a id="CO448-4"></a><i class="conum" data-value="4"></i>
        def last_doc = '';
        def timestamp_latest = 0L;
        for (s in states) {if (s.timestamp_latest &gt; (timestamp_latest))
        {timestamp_latest = s.timestamp_latest; last_doc = s.last_doc;}}
        return last_doc
      """
    }
  }
}</pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO448-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">init_script</code> creates a long type <code class="literal">timestamp_latest</code> and a string type
<code class="literal">last_doc</code> in the <code class="literal">state</code> object.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO448-2"><i class="conum" data-value="2"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">map_script</code> defines <code class="literal">current_date</code> based on the timestamp of the
document, then compares <code class="literal">current_date</code> with <code class="literal">state.timestamp_latest</code>, finally
returns <code class="literal">state.last_doc</code> from the shard. By using <code class="literal">new HashMap(...)</code> you copy
the source document, this is important whenever you want to pass the full source
object from one phase to the next.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO448-3"><i class="conum" data-value="3"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">combine_script</code> returns <code class="literal">state</code> from each shard.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO448-4"><i class="conum" data-value="4"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">reduce_script</code> iterates through the value of <code class="literal">s.timestamp_latest</code>
returned by each shard and returns the document with the latest timestamp
(<code class="literal">last_doc</code>). In the response, the top hit (in other words, the <code class="literal">latest_doc</code>) is
nested below the <code class="literal">latest_doc</code> field.</p>
</td>
</tr>
</table>
</div>
<p>Check the
<a class="xref" href="search-aggregations-metrics-scripted-metric-aggregation.html#scripted-metric-aggregation-scope" title="Scope of scripts">scope of scripts</a>
for detailed explanation on the respective scripts.</p>
<p>You can retrieve the last value in a similar way:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">"aggregations": {
  "latest_value": {
    "scripted_metric": {
      "init_script": "state.timestamp_latest = 0L; state.last_value = ''",
      "map_script": """
        def current_date = doc['date'].getValue().toInstant().toEpochMilli();
        if (current_date &gt; state.timestamp_latest)
        {state.timestamp_latest = current_date;
        state.last_value = params['_source']['value'];}
      """,
      "combine_script": "return state",
      "reduce_script": """
        def last_value = '';
        def timestamp_latest = 0L;
        for (s in states) {if (s.timestamp_latest &gt; (timestamp_latest))
        {timestamp_latest = s.timestamp_latest; last_value = s.last_value;}}
        return last_value
      """
    }
  }
}</pre>
</div>
<h4>
<a id="painless-time-features"></a>Getting time features by using aggregations<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/transform/painless-examples.asciidoc">edit</a>
</h4>
<p>This snippet shows how to extract time based features by using Painless in a
transform. The snippet uses an index where <code class="literal">@timestamp</code> is defined as a <code class="literal">date</code>
type field.</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">"aggregations": {
  "avg_hour_of_day": { <a id="CO449-1"></a><i class="conum" data-value="1"></i>
    "avg":{
      "script": { <a id="CO449-2"></a><i class="conum" data-value="2"></i>
        "source": """
          ZonedDateTime date =  doc['@timestamp'].value; <a id="CO449-3"></a><i class="conum" data-value="3"></i>
          return date.getHour(); <a id="CO449-4"></a><i class="conum" data-value="4"></i>
        """
      }
    }
  },
  "avg_month_of_year": { <a id="CO449-5"></a><i class="conum" data-value="5"></i>
    "avg":{
      "script": { <a id="CO449-6"></a><i class="conum" data-value="6"></i>
        "source": """
          ZonedDateTime date =  doc['@timestamp'].value; <a id="CO449-7"></a><i class="conum" data-value="7"></i>
          return date.getMonthValue(); <a id="CO449-8"></a><i class="conum" data-value="8"></i>
        """
      }
    }
  },
 ...
}</pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO449-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>Name of the aggregation.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO449-2"><i class="conum" data-value="2"></i></a></p>
</td>
<td align="left" valign="top">
<p>Contains the Painless script that returns the hour of the day.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO449-3"><i class="conum" data-value="3"></i></a></p>
</td>
<td align="left" valign="top">
<p>Sets <code class="literal">date</code> based on the timestamp of the document.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO449-4"><i class="conum" data-value="4"></i></a></p>
</td>
<td align="left" valign="top">
<p>Returns the hour value from <code class="literal">date</code>.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO449-5"><i class="conum" data-value="5"></i></a></p>
</td>
<td align="left" valign="top">
<p>Name of the aggregation.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO449-6"><i class="conum" data-value="6"></i></a></p>
</td>
<td align="left" valign="top">
<p>Contains the Painless script that returns the month of the year.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO449-7"><i class="conum" data-value="7"></i></a></p>
</td>
<td align="left" valign="top">
<p>Sets <code class="literal">date</code> based on the timestamp of the document.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO449-8"><i class="conum" data-value="8"></i></a></p>
</td>
<td align="left" valign="top">
<p>Returns the month value from <code class="literal">date</code>.</p>
</td>
</tr>
</table>
</div>
<h4>
<a id="painless-group-by"></a>Using Painless in <code class="literal">group_by</code><a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/transform/painless-examples.asciidoc">edit</a>
</h4>
<p>It is possible to base the <code class="literal">group_by</code> property of a transform on the output of
a script. The following example uses the Kibana sample web logs dataset. The goal
here is to make the transform output easier to understand through normalizing
the value of the fields that the data is grouped by.</p>
<p>If you use scripts in <code class="literal">group_by</code>, the transform cannot deduce the mappings of
the field. For this reason, you need to create the destination index and its
mappings prior to creating the transform. Please refer to the example below.</p>
<p>Create the index:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">PUT /pivot_logs</pre>
</div>
<div class="console_widget" data-snippet="snippets/1162.console"></div>
<p>Create the mapping of the index:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">PUT /pivot_logs/_mapping
{
  "properties": {
    "200": {"type": "long"},
    "404":  { "type": "long"},
    "503":  { "type": "long"},
    "agent":  { "type": "keyword"}
  }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/1163.console"></div>
<p>Then you can create the transform. The example below shows you how to create a
preview of the transform.</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">POST _transform/_preview
{
  "source": {
    "index": [ <a id="CO450-1"></a><i class="conum" data-value="1"></i>
      "kibana_sample_data_logs"
    ]
  },
  "pivot": {
    "group_by": {
      "agent": {
        "terms": {
          "script": { <a id="CO450-2"></a><i class="conum" data-value="2"></i>
            "source": """String agent = doc['agent.keyword'].value;
            if (agent.contains("MSIE")) {
              return "internet explorer";
            } else if (agent.contains("AppleWebKit")) {
              return "safari";
            } else if (agent.contains('Firefox')) {
              return "firefox";
            } else { return agent }""",
            "lang": "painless"
          }
        }
      }
    },
    "aggregations": { <a id="CO450-3"></a><i class="conum" data-value="3"></i>
      "200": {
        "filter": {
          "term": {
            "response": "200"
          }
        }
      },
      "404": {
        "filter": {
          "term": {
            "response": "404"
          }
        }
      },
      "503": {
        "filter": {
          "term": {
            "response": "503"
          }
        }
      }
    }
  },
  "dest": { <a id="CO450-4"></a><i class="conum" data-value="4"></i>
    "index": "pivot_logs"
  }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/1164.console"></div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO450-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>Specifies the source index or indices.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO450-2"><i class="conum" data-value="2"></i></a></p>
</td>
<td align="left" valign="top">
<p>The script defines an <code class="literal">agent</code> string based on the <code class="literal">agent</code> field of the
documents, then iterates through the values. If an <code class="literal">agent</code> field contains
"MSIE", than the script returns "Internet Explorer". If it contains
<code class="literal">AppleWebKit</code>, it returns "safari". It returns "firefox" if the field value
contains "Firefox". Finally, in every other case, the value of the field is
returned.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO450-3"><i class="conum" data-value="3"></i></a></p>
</td>
<td align="left" valign="top">
<p>The aggregations object contains filters that narrow down the results to
documents that contains <code class="literal">200</code>, <code class="literal">404</code>, or <code class="literal">503</code> values in the <code class="literal">response</code> field.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO450-4"><i class="conum" data-value="4"></i></a></p>
</td>
<td align="left" valign="top">
<p>Specifies the destination index of the transform. As the mappings of
fields created by scrips cannot be deduced, please define the mappings of the
destination index prior to creating the transform.</p>
</td>
</tr>
</table>
</div>
<p>The API returns the following result:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">{
  "preview" : [
    {
      "agent" : "firefox",
      "200" : 4931,
      "404" : 259,
      "503" : 172
    },
    {
      "agent" : "internet explorer",
      "200" : 3674,
      "404" : 210,
      "503" : 126
    },
    {
      "agent" : "safari",
      "200" : 4227,
      "404" : 332,
      "503" : 143
    }
  ],
  "mappings" : {
    "properties" : {
      "200" : {
        "type" : "long"
      },
      "agent" : {
        "type" : "keyword"
      },
      "404" : {
        "type" : "long"
      },
      "503" : {
        "type" : "long"
      }
    }
  }
}</pre>
</div>
<p>You can see that the <code class="literal">agent</code> values are simplified so it is easier to interpret
them. The table below shows how normalization modifies the output of the
transform in our example compared to the non-normalized values.</p>
<div class="informaltable">
<table border="1" cellpadding="4px" width="50%">
<colgroup>
<col class="col_1">
<col class="col_2">
</colgroup>
<thead>
<tr>
<th align="left" valign="top">Non-normalized <code class="literal">agent</code> value</th>
<th align="left" valign="top">Normalized <code class="literal">agent</code> value</th>
</tr>
</thead>
<tbody>
<tr>
<td align="left" valign="top"><p>"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)"</p></td>
<td align="left" valign="top"><p>"internet explorer"</p></td>
</tr>
<tr>
<td align="left" valign="top"><p>"Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24"</p></td>
<td align="left" valign="top"><p>"safari"</p></td>
</tr>
<tr>
<td align="left" valign="top"><p>"Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1"</p></td>
<td align="left" valign="top"><p>"firefox"</p></td>
</tr>
</tbody>
</table>
</div>
<h4>
<a id="painless-bucket-script"></a>Getting duration by using bucket script<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/transform/painless-examples.asciidoc">edit</a>
</h4>
<p>This example shows you how to get the duration of a session by client IP from a
data log by using
<a href="/guide/en/elasticsearch/reference/7.7/search-aggregations-pipeline-bucket-script-aggregation.html" class="ulink" target="_top">bucket script</a>.
The example uses the Kibana sample web logs dataset.</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">PUT _transform/data_log
{
  "source": {
    "index": "kibana_sample_data_logs"
  },
  "dest": {
    "index": "data-logs-by-client"
  },
  "pivot": {
    "group_by": {
      "machine.os": {"terms": {"field": "machine.os.keyword"}},
      "machine.ip": {"terms": {"field": "clientip"}}
    },
    "aggregations": {
      "time_frame.lte": {
        "max": {
          "field": "timestamp"
        }
      },
      "time_frame.gte": {
        "min": {
          "field": "timestamp"
        }
      },
      "time_length": { <a id="CO451-1"></a><i class="conum" data-value="1"></i>
        "bucket_script": {
          "buckets_path": { <a id="CO451-2"></a><i class="conum" data-value="2"></i>
            "min": "time_frame.gte.value",
            "max": "time_frame.lte.value"
          },
          "script": "params.max - params.min" <a id="CO451-3"></a><i class="conum" data-value="3"></i>
        }
      }
    }
  }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/1165.console"></div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO451-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>To define the length of the sessions, we use a bucket script.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO451-2"><i class="conum" data-value="2"></i></a></p>
</td>
<td align="left" valign="top">
<p>The bucket path is a map of script variables and their associated path to
the buckets you want to use for the variable. In this particular case, <code class="literal">min</code> and
<code class="literal">max</code> are variables mapped to <code class="literal">time_frame.gte.value</code> and <code class="literal">time_frame.lte.value</code>.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO451-3"><i class="conum" data-value="3"></i></a></p>
</td>
<td align="left" valign="top">
<p>Finally, the script substracts the start date of the session from the end
date which results in the duration of the session.</p>
</td>
</tr>
</table>
</div>
<h4>
<a id="painless-count-http"></a>Counting HTTP responses by using scripted metric aggregation<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/transform/painless-examples.asciidoc">edit</a>
</h4>
<p>You can count the different HTTP response types in a web log data set by using
scripted metric aggregation as part of the transform. The example below
assumes that the HTTP response codes are stored as keywords in the <code class="literal">response</code>
field of the documents.</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">"aggregations": { <a id="CO452-1"></a><i class="conum" data-value="1"></i>
  "responses.counts": { <a id="CO452-2"></a><i class="conum" data-value="2"></i>
    "scripted_metric": { <a id="CO452-3"></a><i class="conum" data-value="3"></i>
      "init_script": "state.responses = ['error':0L,'success':0L,'other':0L]", <a id="CO452-4"></a><i class="conum" data-value="4"></i>
      "map_script": """ <a id="CO452-5"></a><i class="conum" data-value="5"></i>
        def code = doc['response.keyword'].value;
        if (code.startsWith('5') || code.startsWith('4')) {
          state.responses.error += 1 ;
        } else if(code.startsWith('2')) {
          state.responses.success += 1;
        } else {
          state.responses.other += 1;
        }
        """,
      "combine_script": "state.responses", <a id="CO452-6"></a><i class="conum" data-value="6"></i>
      "reduce_script": """ <a id="CO452-7"></a><i class="conum" data-value="7"></i>
        def counts = ['error': 0L, 'success': 0L, 'other': 0L];
        for (responses in states) {
          counts.error += responses['error'];
          counts.success += responses['success'];
          counts.other += responses['other'];
        }
        return counts;
        """
      }
    },
  ...
}</pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO452-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">aggregations</code> object of the transform that contains all aggregations.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO452-2"><i class="conum" data-value="2"></i></a></p>
</td>
<td align="left" valign="top">
<p>Object of the <code class="literal">scripted_metric</code> aggregation.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO452-3"><i class="conum" data-value="3"></i></a></p>
</td>
<td align="left" valign="top">
<p>This <code class="literal">scripted_metric</code> performs a distributed operation on the web log data
to count specific types of HTTP responses (error, success, and other).</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO452-4"><i class="conum" data-value="4"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">init_script</code> creates a <code class="literal">responses</code> array in the <code class="literal">state</code> object with
three properties (<code class="literal">error</code>, <code class="literal">success</code>, <code class="literal">other</code>) with long data type.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO452-5"><i class="conum" data-value="5"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">map_script</code> defines <code class="literal">code</code> based on the <code class="literal">response.keyword</code> value of the
document, then it counts the errors, successes, and other responses based on the
first digit of the responses.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO452-6"><i class="conum" data-value="6"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">combine_script</code> returns <code class="literal">state.responses</code> from each shard.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO452-7"><i class="conum" data-value="7"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">reduce_script</code> creates a <code class="literal">counts</code> array with the <code class="literal">error</code>, <code class="literal">success</code>,
and <code class="literal">other</code> properties, then iterates through the value of <code class="literal">responses</code> returned
by each shard and assigns the different response types to the appropriate
properties of the <code class="literal">counts</code> object; error responses to the error counts, success
responses to the success counts, and other responses to the other counts.
Finally, returns the <code class="literal">counts</code> array with the response counts.</p>
</td>
</tr>
</table>
</div>
<h4>
<a id="painless-compare"></a>Comparing indices by using scripted metric aggregations<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/transform/painless-examples.asciidoc">edit</a>
</h4>
<p>This example shows how to compare the content of two indices by a transform
that uses a scripted metric aggregation.</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">POST _transform/_preview
{
  "id" : "index_compare",
  "source" : { <a id="CO453-1"></a><i class="conum" data-value="1"></i>
    "index" : [
      "index1",
      "index2"
    ],
    "query" : {
      "match_all" : { }
    }
  },
  "dest" : { <a id="CO453-2"></a><i class="conum" data-value="2"></i>
    "index" : "compare"
  },
  "pivot" : {
    "group_by" : {
      "unique-id" : {
        "terms" : {
          "field" : "&lt;unique-id-field&gt;" <a id="CO453-3"></a><i class="conum" data-value="3"></i>
        }
      }
    },
    "aggregations" : {
      "compare" : { <a id="CO453-4"></a><i class="conum" data-value="4"></i>
        "scripted_metric" : {
          "init_script" : "",
          "map_script" : "state.doc = new HashMap(params['_source'])", <a id="CO453-5"></a><i class="conum" data-value="5"></i>
          "combine_script" : "return state", <a id="CO453-6"></a><i class="conum" data-value="6"></i>
          "reduce_script" : """ <a id="CO453-7"></a><i class="conum" data-value="7"></i>
            if (states.size() != 2) {
              return "count_mismatch"
            }
            if (states.get(0).equals(states.get(1))) {
              return "match"
            } else {
              return "mismatch"
            }
            """
        }
      }
    }
  }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/1166.console"></div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO453-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>The indices referenced in the <code class="literal">source</code> object are compared to each other.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO453-2"><i class="conum" data-value="2"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">dest</code> index contains the results of the comparison.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO453-3"><i class="conum" data-value="3"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">group_by</code> field needs to be a unique identifier for each document.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO453-4"><i class="conum" data-value="4"></i></a></p>
</td>
<td align="left" valign="top">
<p>Object of the <code class="literal">scripted_metric</code> aggregation.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO453-5"><i class="conum" data-value="5"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">map_script</code> defines <code class="literal">doc</code> in the state object. By using
<code class="literal">new HashMap(...)</code> you copy the source document, this is important whenever you
want to pass the full source object from one phase to the next.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO453-6"><i class="conum" data-value="6"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">combine_script</code> returns <code class="literal">state</code> from each shard.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO453-7"><i class="conum" data-value="7"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">reduce_script</code> checks if the size of the indices are equal. If they are
not equal, than it reports back a <code class="literal">count_mismatch</code>. Then it iterates through all
the values of the two indices and compare them. If the values are equal, then it
returns a <code class="literal">match</code>, otherwise returns a <code class="literal">mismatch</code>.</p>
</td>
</tr>
</table>
</div>
<h4>
<a id="painless-web-session"></a>Getting web session details by using scripted metric aggregation<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/transform/painless-examples.asciidoc">edit</a>
</h4>
<p>This example shows how to derive multiple features from a single transaction.
Let’s take a look on the example source document from the data:</p>
<details open>
<summary class="title">Source document</summary>
<div class="content">
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">{
  "_index":"apache-sessions",
  "_type":"_doc",
  "_id":"KvzSeGoB4bgw0KGbE3wP",
  "_score":1.0,
  "_source":{
    "@timestamp":1484053499256,
    "apache":{
      "access":{
        "sessionid":"571604f2b2b0c7b346dc685eeb0e2306774a63c2",
        "url":"http://www.leroymerlin.fr/v3/search/search.do?keyword=Carrelage%20salle%20de%20bain",
        "path":"/v3/search/search.do",
        "query":"keyword=Carrelage%20salle%20de%20bain",
        "referrer":"http://www.leroymerlin.fr/v3/p/produits/carrelage-parquet-sol-souple/carrelage-sol-et-mur/decor-listel-et-accessoires-carrelage-mural-l1308217717?resultOffset=0&amp;resultLimit=51&amp;resultListShape=MOSAIC&amp;priceStyle=SALEUNIT_PRICE",
        "user_agent":{
          "original":"Mobile Safari 10.0 Mac OS X (iPad) Apple Inc.",
          "os_name":"Mac OS X (iPad)"
        },
        "remote_ip":"0337b1fa-5ed4-af81-9ef4-0ec53be0f45d",
        "geoip":{
          "country_iso_code":"FR",
          "location":{
            "lat":48.86,
            "lon":2.35
          }
        },
        "response_code":200,
        "method":"GET"
      }
    }
  }
}
...</pre>
</div>
</div>
</details>
<p>By using the <code class="literal">sessionid</code> as a group-by field, you are able to enumerate events
through the session and get more details of the session by using scripted metric
aggregation.</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">POST _transform/_preview
{
  "source": {
    "index": "apache-sessions"
  },
  "pivot": {
    "group_by": {
      "sessionid": { <a id="CO454-1"></a><i class="conum" data-value="1"></i>
        "terms": {
          "field": "apache.access.sessionid"
        }
      }
    },
    "aggregations": { <a id="CO454-2"></a><i class="conum" data-value="2"></i>
      "distinct_paths": {
        "cardinality": {
          "field": "apache.access.path"
        }
      },
      "num_pages_viewed": {
        "value_count": {
          "field": "apache.access.url"
        }
      },
      "session_details": {
        "scripted_metric": {
          "init_script": "state.docs = []", <a id="CO454-3"></a><i class="conum" data-value="3"></i>
          "map_script": """ <a id="CO454-4"></a><i class="conum" data-value="4"></i>
            Map span = [
              '@timestamp':doc['@timestamp'].value,
              'url':doc['apache.access.url'].value,
              'referrer':doc['apache.access.referrer'].value
            ];
            state.docs.add(span)
          """,
          "combine_script": "return state.docs;", <a id="CO454-5"></a><i class="conum" data-value="5"></i>
          "reduce_script": """ <a id="CO454-6"></a><i class="conum" data-value="6"></i>
            def all_docs = [];
            for (s in states) {
              for (span in s) {
                all_docs.add(span);
              }
            }
            all_docs.sort((HashMap o1, HashMap o2)-&gt;o1['@timestamp'].millis.compareTo(o2['@timestamp'].millis));
            def size = all_docs.size();
            def min_time = all_docs[0]['@timestamp'];
            def max_time = all_docs[size-1]['@timestamp'];
            def duration = max_time.millis - min_time.millis;
            def entry_page = all_docs[0]['url'];
            def exit_path = all_docs[size-1]['url'];
            def first_referrer = all_docs[0]['referrer'];
            def ret = new HashMap();
            ret['first_time'] = min_time;
            ret['last_time'] = max_time;
            ret['duration'] = duration;
            ret['entry_page'] = entry_page;
            ret['exit_path'] = exit_path;
            ret['first_referrer'] = first_referrer;
            return ret;
          """
        }
      }
    }
  }
}</pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO454-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>The data is grouped by <code class="literal">sessionid</code>.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO454-2"><i class="conum" data-value="2"></i></a></p>
</td>
<td align="left" valign="top">
<p>The aggregations counts the number of paths and enumerate the viewed pages
during the session.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO454-3"><i class="conum" data-value="3"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">init_script</code> creates an array type <code class="literal">doc</code> in the <code class="literal">state</code> object.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO454-4"><i class="conum" data-value="4"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">map_script</code> defines a <code class="literal">span</code> array with a timestamp, a URL, and a
referrer value which are based on the corresponding values of the document, then
adds the value of the <code class="literal">span</code> array to the <code class="literal">doc</code> object.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO454-5"><i class="conum" data-value="5"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">combine_script</code> returns <code class="literal">state.docs</code> from each shard.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO454-6"><i class="conum" data-value="6"></i></a></p>
</td>
<td align="left" valign="top">
<p>The <code class="literal">reduce_script</code> defines various objects like <code class="literal">min_time</code>, <code class="literal">max_time</code>, and
<code class="literal">duration</code> based on the document fields, then declares a <code class="literal">ret</code> object, and
copies the source document by using <code class="literal">new HashMap ()</code>. Next, the script defines
<code class="literal">first_time</code>, <code class="literal">last_time</code>, <code class="literal">duration</code> and other fields inside the <code class="literal">ret</code> object
based on the corresponding object defined earlier, finally returns <code class="literal">ret</code>.</p>
</td>
</tr>
</table>
</div>
<p>The API call results in a similar response:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">{
  "num_pages_viewed" : 2.0,
  "session_details" : {
    "duration" : 131374,
    "first_referrer" : "https://www.bing.com/",
    "entry_page" : "http://www.leroymerlin.fr/v3/p/produits/materiaux-menuiserie/porte-coulissante-porte-interieure-escalier-et-rambarde/barriere-de-securite-l1308218463",
    "first_time" : "2017-01-10T21:22:52.982Z",
    "last_time" : "2017-01-10T21:25:04.356Z",
    "exit_path" : "http://www.leroymerlin.fr/v3/p/produits/materiaux-menuiserie/porte-coulissante-porte-interieure-escalier-et-rambarde/barriere-de-securite-l1308218463?__result-wrapper?pageTemplate=Famille%2FMat%C3%A9riaux+et+menuiserie&amp;resultOffset=0&amp;resultLimit=50&amp;resultListShape=PLAIN&amp;nomenclatureId=17942&amp;priceStyle=SALEUNIT_PRICE&amp;fcr=1&amp;*4294718806=4294718806&amp;*14072=14072&amp;*4294718593=4294718593&amp;*17942=17942"
  },
  "distinct_paths" : 1.0,
  "sessionid" : "000046f8154a80fd89849369c984b8cc9d795814"
},
{
  "num_pages_viewed" : 10.0,
  "session_details" : {
    "duration" : 343112,
    "first_referrer" : "https://www.google.fr/",
    "entry_page" : "http://www.leroymerlin.fr/",
    "first_time" : "2017-01-10T16:57:39.937Z",
    "last_time" : "2017-01-10T17:03:23.049Z",
    "exit_path" : "http://www.leroymerlin.fr/v3/p/produits/porte-de-douche-coulissante-adena-e168578"
  },
  "distinct_paths" : 8.0,
  "sessionid" : "000087e825da1d87a332b8f15fa76116c7467da6"
}
...</pre>
</div>
</div>
<div class="navfooter">
<span class="prev">
<a href="transform-examples.html">« Transform examples</a>
</span>
<span class="next">
<a href="transform-troubleshooting.html">Troubleshooting transforms »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
